{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-07T04:59:56.218000Z",
     "start_time": "2018-01-07T04:59:56.204000Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-07T04:59:57.725000Z",
     "start_time": "2018-01-07T04:59:56.440000Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-07T04:59:57.785000Z",
     "start_time": "2018-01-07T04:59:57.729000Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-07T04:59:57.813000Z",
     "start_time": "2018-01-07T04:59:57.789000Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 49352 entries, 0 to 49351\n",
      "Columns: 228 entries, bathrooms to interest_level\n",
      "dtypes: float64(9), int64(219)\n",
      "memory usage: 85.8 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-07T04:59:58.643000Z",
     "start_time": "2018-01-07T04:59:57.817000Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>49352.00000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>4.935200e+04</td>\n",
       "      <td>4.935200e+04</td>\n",
       "      <td>4.935200e+04</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.0</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.21218</td>\n",
       "      <td>1.541640</td>\n",
       "      <td>3.830174e+03</td>\n",
       "      <td>1.697863e+03</td>\n",
       "      <td>1.657567e+03</td>\n",
       "      <td>-0.329460</td>\n",
       "      <td>2.753820</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>5.014852</td>\n",
       "      <td>15.206881</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003080</td>\n",
       "      <td>0.000385</td>\n",
       "      <td>0.186477</td>\n",
       "      <td>0.009361</td>\n",
       "      <td>0.000446</td>\n",
       "      <td>0.028165</td>\n",
       "      <td>0.002026</td>\n",
       "      <td>0.001013</td>\n",
       "      <td>0.000952</td>\n",
       "      <td>1.616895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.50142</td>\n",
       "      <td>1.115018</td>\n",
       "      <td>2.206687e+04</td>\n",
       "      <td>1.100477e+04</td>\n",
       "      <td>7.817996e+03</td>\n",
       "      <td>0.947732</td>\n",
       "      <td>1.446091</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.824442</td>\n",
       "      <td>8.280749</td>\n",
       "      <td>...</td>\n",
       "      <td>0.055412</td>\n",
       "      <td>0.019618</td>\n",
       "      <td>0.389495</td>\n",
       "      <td>0.101625</td>\n",
       "      <td>0.021109</td>\n",
       "      <td>0.165446</td>\n",
       "      <td>0.044969</td>\n",
       "      <td>0.031814</td>\n",
       "      <td>0.030846</td>\n",
       "      <td>0.626035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.300000e+01</td>\n",
       "      <td>2.150000e+01</td>\n",
       "      <td>4.300000e+01</td>\n",
       "      <td>-5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.500000e+03</td>\n",
       "      <td>1.225000e+03</td>\n",
       "      <td>1.066667e+03</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.150000e+03</td>\n",
       "      <td>1.500000e+03</td>\n",
       "      <td>1.383417e+03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.100000e+03</td>\n",
       "      <td>1.850000e+03</td>\n",
       "      <td>1.962500e+03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>10.00000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>4.490000e+06</td>\n",
       "      <td>2.245000e+06</td>\n",
       "      <td>1.496667e+06</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>13.500000</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>31.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         bathrooms      bedrooms         price  price_bathrooms  \\\n",
       "count  49352.00000  49352.000000  4.935200e+04     4.935200e+04   \n",
       "mean       1.21218      1.541640  3.830174e+03     1.697863e+03   \n",
       "std        0.50142      1.115018  2.206687e+04     1.100477e+04   \n",
       "min        0.00000      0.000000  4.300000e+01     2.150000e+01   \n",
       "25%        1.00000      1.000000  2.500000e+03     1.225000e+03   \n",
       "50%        1.00000      1.000000  3.150000e+03     1.500000e+03   \n",
       "75%        1.00000      2.000000  4.100000e+03     1.850000e+03   \n",
       "max       10.00000      8.000000  4.490000e+06     2.245000e+06   \n",
       "\n",
       "       price_bedrooms     room_diff      room_num     Year         Month  \\\n",
       "count    4.935200e+04  49352.000000  49352.000000  49352.0  49352.000000   \n",
       "mean     1.657567e+03     -0.329460      2.753820   2016.0      5.014852   \n",
       "std      7.817996e+03      0.947732      1.446091      0.0      0.824442   \n",
       "min      4.300000e+01     -5.000000      0.000000   2016.0      4.000000   \n",
       "25%      1.066667e+03     -1.000000      2.000000   2016.0      4.000000   \n",
       "50%      1.383417e+03      0.000000      2.000000   2016.0      5.000000   \n",
       "75%      1.962500e+03      0.000000      4.000000   2016.0      6.000000   \n",
       "max      1.496667e+06      8.000000     13.500000   2016.0      6.000000   \n",
       "\n",
       "                Day       ...                walk         walls           war  \\\n",
       "count  49352.000000       ...        49352.000000  49352.000000  49352.000000   \n",
       "mean      15.206881       ...            0.003080      0.000385      0.186477   \n",
       "std        8.280749       ...            0.055412      0.019618      0.389495   \n",
       "min        1.000000       ...            0.000000      0.000000      0.000000   \n",
       "25%        8.000000       ...            0.000000      0.000000      0.000000   \n",
       "50%       15.000000       ...            0.000000      0.000000      0.000000   \n",
       "75%       22.000000       ...            0.000000      0.000000      0.000000   \n",
       "max       31.000000       ...            1.000000      1.000000      1.000000   \n",
       "\n",
       "             washer         water    wheelchair          wifi       windows  \\\n",
       "count  49352.000000  49352.000000  49352.000000  49352.000000  49352.000000   \n",
       "mean       0.009361      0.000446      0.028165      0.002026      0.001013   \n",
       "std        0.101625      0.021109      0.165446      0.044969      0.031814   \n",
       "min        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "max        2.000000      1.000000      1.000000      1.000000      1.000000   \n",
       "\n",
       "               work  interest_level  \n",
       "count  49352.000000    49352.000000  \n",
       "mean       0.000952        1.616895  \n",
       "std        0.030846        0.626035  \n",
       "min        0.000000        0.000000  \n",
       "25%        0.000000        1.000000  \n",
       "50%        0.000000        2.000000  \n",
       "75%        0.000000        2.000000  \n",
       "max        1.000000        2.000000  \n",
       "\n",
       "[8 rows x 228 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-07T04:59:58.897000Z",
     "start_time": "2018-01-07T04:59:58.892000Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# sns.countplot(train.interest_level);\n",
    "# pyplot.xlabel('interest_level');\n",
    "# pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-07T05:00:00.512000Z",
     "start_time": "2018-01-07T05:00:00.335000Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train=train['interest_level']\n",
    "train = train.drop([\"interest_level\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-07T05:00:02.399000Z",
     "start_time": "2018-01-07T05:00:02.388000Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)\n",
    "\n",
    "#params = {\"objective\": \"multi:softprob\", \"eval_metric\":\"mlogloss\", \"num_class\": 9}\n",
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=219,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-07T05:00:05.495000Z",
     "start_time": "2018-01-07T05:00:05.474000Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#直接调用xgboost内嵌的交叉验证（cv），可对连续的n_estimators参数进行快速交叉验证\n",
    "#而GridSearchCV只能对有限个参数进行交叉验证\n",
    "def modelfit(alg, X_train, y_train, cv_folds=None, early_stopping_rounds=100):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 3\n",
    "    \n",
    "    #直接调用xgboost，而非sklarn的wrapper类\n",
    "    xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "  \n",
    "    cvresult.to_csv('2_nestimators.csv', index_label = 'n_estimators')\n",
    "    \n",
    "    #最佳参数n_estimators\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "#     Predict training set:\n",
    "    train_predprob = alg.predict_proba(X_train)\n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "#    Print model report:\n",
    "    print (\"logloss of train :\")\n",
    "    print logloss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-07T05:11:52.412000Z",
     "start_time": "2018-01-07T05:00:06.827000Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of train :\n",
      "0.522325989914\n"
     ]
    }
   ],
   "source": [
    "\n",
    "modelfit(xgb1, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-07T05:12:44.938000Z",
     "start_time": "2018-01-07T05:12:44.665000Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8ZGWZ//3PVZV966Q76X2nu9mx\nwWZHQUEFVFB2FBRcUB8Z9/EH8/hzGJ5hBndBGRAUehQEEZcBZdzYlJ1mb2gaet/TazqdPam6nj/u\nk3R1qHSnm1ROkvq+X6/zStWpU6euOknqW/e5z7mPuTsiIiIAibgLEBGRoUOhICIiPRQKIiLSQ6Eg\nIiI9FAoiItJDoSAiIj0UCiIZzOxfzOyncdchEheFwjBjZhVmtsLMPpIxr9LMVpnZORnz5pnZH8xs\nm5k1mNmrZnaNmdVEj19iZikza4qmZWb2uRzXfpKZrcnla+yNbPW4+3+4+6dy9HorzOyUXKw7Fwbr\n9zXctstIp1AYZty9CbgMuM7M6qLZ3wYWuPs9AGZ2HPAw8BhwgLtXA6cCXcDbMlb3hLtXuHsFcA7w\nbTM7fHDeiewNMyuIuwbJE+6uaRhOwHzgTuAkYAswIeOxR4Ef7eH5lwCP9pr3NPCRjPtnAK8ADYSQ\nOTDjsQOjeQ3RMmdkPHY68CqwA1gLfA0oB1qBNNAUTRP7eF83AH+Mnv8UsF8/tscBwF+BrcBi4Lx9\nqQe4Crg9et50wIFLgdXANuCzwJHAS9F7/3HG6+wHPBj9PjYDdwDV0WO/iF6rNXqtr/djG68A/k/0\nWu1AQXR/bfReFgMnZ9kWxwAbgGTGvA8DL0W3jwIWAI1APfD9PrbpScCaPh4bBfwc2ASsBL4BJKLH\nksD3om2wHLg82o4FfaxrBXBKH499GlgS/V7v7f6bAQz4AbAR2B5to0P6+n3H/f86nKbYC9C0j784\nqAHWR/94l2bMLwdSwEl7eP4lZIRC9EHXAMyJ7s8BmoH3AIXA16N/zqLo/hLgX6L7747+AfePnrse\neEdGnUdEt/v8kMmoY370AXBU9CF4B3DXHp5TTvjQvjR6zhHRdjl4b+sheyjcBJQA7wXagN8DY4FJ\n0YfSidHys6LtVQzUAX8Hfpix7l0+/Ha3jTOWfwGYApQC+0fvc2JGfVkDE1gKvCfj/q+BK6LbTwAX\nR7crgGP6WEefvy9CIPwPUBnV8TrwyeixzxI+lCdH2/tv7EMoRH9Xm6PfZzHwI+Dv0WPvA54FqgkB\ncSDRF6O+ft+a+jdp99Ew5e7bCN8wy4DfZjxUQ9gtuKF7hpl9O+pXaDazb2Qse0w0v4nQSvgF8Eb0\n2PnAH939r+7eCXyX8MF0HOGbaAVwrbt3uPuDwB+AC6PndgIHmVmVu29z9+f28u391t2fdvcuQijM\n3cPyHwBWuPtt7t4Vvd5vCLvEBqKe/8/d29z9L4QP8TvdfaO7rwX+ARwO4O5Lou3V7u6bgO8DJ+5m\nvbvbxt2ud/fV7t5KCPvi6L0UuvsKd1/ax7rvJPp9mFkl4dvznRnbY5aZ1bp7k7s/uTcbw8ySUe1X\nuvsOd19BaBlcHC1yHnCdu6+J/k6v3Zv1Z/gocKu7P+fu7cCVwLFmNj16D5WEFqK5+yJ3X5/x/t7K\n7zuvKRSGKTO7iPAN7W/AtzIe2kbYTTGhe4a7f91Dv8LvCN+kuz3p7tUe+hTGAwcD/xE9NpGwW6B7\nHWnCt9RJ0WOro3ndVkaPAZxN+BBaaWaPmNmxe/n2NmTcbiEE0O5MA46OAq7BzBoIHyjjB6ie+ozb\nrVnuVwCY2Vgzu8vM1ppZI3A7ULub9e5uG3dbnfH4EuBLhNbMxui1Jvax7l8CZ5lZMXAW8Jy7d7/W\nJwmtlNfM7Bkz+8BuasymltBCXJkxL/P3PzGz7l6390bv7dNE2DU3Kfoi8mPCrsZ6M7vZzKqiRd/q\n7zuvKRSGITMbS9if+mngM8B5ZvZOAHdvJuyHP2tv1unu9YRv1x+MZq0jfNh2v6YRdmOsjR6bYmaZ\nfz9To8dw92fc/UzCLpbfA3d3v8ze1LQXVgOPRAHXPVW4++cGuZ7/jNZ5mLtXARcRdm106/16u9vG\nWZ/j7r909xOi5zm7fiHIXO5VwgfqacBHCCHR/dgb7n4hYXt8C7jHzMr7/zbZTPg2Pi1jXs/vn7D7\nZnLGY1P2Yt2Zem+fcmAMO//Ornf3txO+zMwB/jma39fvW/pBoTA8/Rj4vbs/FDWZvw7cEn0rJLr/\nCTO7IgoQzGwyMKOvFZrZGEJn5CvRrLuB95vZyWZWCHyV0Nn5OCF0moGvm1mhmZ1ECJO7zKzIzD5q\nZqOiXSKNhN0eEL5hjzGzUQO0Hbr9AZhjZhdH9RSa2ZFmduAg11NJ6ERuMLNJRB9SGeqBmRn3d7eN\n38TM9jezd0e/5zZCKyWVbdnIL4EvAO8k9Cl0r+ciM6uLWiYN0ew+12NmJZkToSV6N3BNdDj0NOAr\nhJZR9/v6oplNMrNqQuf4nhT2ep2CqP5LzWxu9J7/A3jK3VdEv9+jo+3WHG2P1B5+39IfcXdqaNq7\nCfgQ4RtUda/5DwDXZNw/Grif8E/fACwErgHGRI9fQvhn6T7yZiNhn/PYjHV8mNBhuB14hKjjNnrs\n4Gje9miZD0fzi4A/EXZjNQLPACdkPO9Wwi6ABvo++ujfM+6fxB46p6Pl9iccsbQpWv+DhL6IvaqH\n7B3NBRnLryGjE5/wQfiNjG3ybLQ9XyB8yK/JWPZMYFX0Wl/rxzZewa4d04cR+n52EDrj/5BtG2Ys\nP5XwAf7HXvNvj37fTYQvAR/q4/knRe+/9zSL0Hd1e7S9VwPfZOfRRwWEluwWwtFHXya0LKyP11mR\n5TX+PXrss4RO8+73OzmafzLhiKMmdh7pVbGn37emPU8WbWARkZwws9OAm9x92h4Xlthp95GIDCgz\nKzWz082sINqN9q+EgxxkGFBLQYYFM3sH8L/ZHvNw9JQMEWZWRtgVdgCh3+OPwBfdvTHWwqRfFAoi\nItJDu49ERKTHsBtkq7a21qdPnx53GSIiw8qzzz672d3r9rTcsAuF6dOns2DBgrjLEBEZVsxs5Z6X\n0u4jERHJoFAQEZEeCgUREemhUBARkR4KBRER6aFQEBGRHgoFERHpkTehsKx+O/c//Qoa1kNEpG95\nEwp/u+VKTr//OJpbmuMuRURkyMqbUDjiwP0AaNxSv4clRUTyV96EQmFlGPKjadvGmCsRERm68iYU\nSqpqAWhpUCiIiPQlb0KhrDq0FDp2bI65EhGRoStvQqGyZhwAXU0KBRGRvuRPKIwOoeDNW2KuRERk\n6MqbUEgWFtNMCbRti7sUEZEhK29CAaDRqihUKIiI9CmvQqE5OYqijoa4yxARGbLyKhTaCkdR0rU9\n7jJERIasvAqFjqJqKlIKBRGRvuQsFMzsVjPbaGYL+3jczOx6M1tiZi+Z2RG5qqVbuqSGKt+hQfFE\nRPqQy5bCfODU3Tx+GjA7mi4DbsxhLQCkS8dQZS20trXl+qVERIalnIWCu/8d2LqbRc4Efu7Bk0C1\nmU3IVT0AyfLRADRs0VAXIiLZxNmnMAlYnXF/TTTvTczsMjNbYGYLNm3atM8vWFgRxj9q1vhHIiJZ\nxRkKlmVe1p397n6zu89z93l1dXX7/ILFo8JzNSieiEh2cYbCGmBKxv3JwLpcvmDpqLEAtDdq/CMR\nkWziDIV7gY9FRyEdA2x39/W5fMHKGo2UKiKyOwW5WrGZ3QmcBNSa2RrgX4FCAHe/CbgfOB1YArQA\nl+aqlm5VY0I/tjfte7+EiMhIlrNQcPcL9/C4A5/P1etnkywuo5kSrEUtBRGRbPLqjGaA7YkaCtsU\nCiIi2eRdKDQV1FDasbvTJ0RE8lfehUJb0WgquhQKIiLZ5F0odJXWMiq9XeMfiYhkkXeh4OV11LCD\n5rb2uEsRERly8i4UEhVjSZizbWNOT4kQERmW8i4UikaNA6BxS05PnhYRGZbyLhRKa8IJbC3b1FIQ\nEekt70KhqnYiAO3b62OuRERk6Mm7UBhVF0bnTu3QSKkiIr3lXSgUllXTQQHWrFAQEekt70IBMxqs\nmoLWLXFXIiIy5ORfKBCGuihpVyiIiPSWl6GwrrOSIoWCiMib5GUoVNROZCwNGupCRKSXvAyFdMUE\nxtDA9ubWuEsRERlS8jIUCqonkjRn84bVcZciIjKk5GUolI6ZDMD2jatirkREZGjJy1CorJsKQOuW\nNTFXIiIytORlKIwePw2Azm0aFE9EJFNehkJR1Ti6SECTBsUTEcmUl6FAIsE2G01RswbFExHJlJ+h\nADQW1lLavinuMkREhpS8DYXWkrGM6lIoiIhkyttQ6CofT51vpaMrHXcpIiJDRt6GglVOoMpa2LR1\na9yliIgMGXkbCn9ZbQBs3bAy5kpERIaOvA2FC085BoDGeoWCiEi3nIaCmZ1qZovNbImZXZHl8Wlm\n9oCZvWRmD5vZ5FzWk2nMpJkAtG5WKIiIdMtZKJhZErgBOA04CLjQzA7qtdh3gZ+7+2HA1cB/5qqe\n3krHhLOaadCgeCIi3XLZUjgKWOLuy9y9A7gLOLPXMgcBD0S3H8ryeO4UlrDVaihsWjtoLykiMtTl\nMhQmAZlfw9dE8zK9CJwd3f4wUGlmY3qvyMwuM7MFZrZg06aBO7dge9F4Kto0/pGISLdchoJlmdf7\nUmdfA040s+eBE4G1QNebnuR+s7vPc/d5dXV1A1ZgS9lExnRt1BXYREQiuQyFNcCUjPuTgV2+lrv7\nOnc/y90PB/7faN72HNa0i3TVZCawhS1NbYP1kiIiQ1ouQ+EZYLaZzTCzIuAC4N7MBcys1sy6a7gS\nuDWH9bxJ4eipFFsn9evU2SwiAjkMBXfvAi4H/gwsAu5291fM7GozOyNa7CRgsZm9DowDrslVPdmU\njZ0BQMP6ZYP5siIiQ1ZBLlfu7vcD9/ea982M2/cA9+Syht2pmbgfAK2blsdVgojIkJK3ZzQDVI4L\nLYXUNl2rWUQE8jwUKBlFk5WT3KFrNYuIQL6HArCtaAIVLTqBTUQEFAq0lE+hrms9XSldV0FEJO9D\nIV09nclsZP225rhLERGJXd6HQlHdLIqtiw1rdASSiEjeh8KoSXMAaFi7OOZKRETil/ehMHpyCIWO\nTUtjrkREJH55HwqJ6il0UkCiYUXcpYiIxC7vQ4FEki0F4yhv1glsIiIKBaCpbApjOtZpCG0RyXsK\nBaCrejpTqGd9Q2vcpYiIxEqhABSPnUWVtbBytXYhiUh+UygANVMPAWDryldirkREJF4KBWDUlAMB\naK9/LeZKRETipVAAbNRU2imiaNuSuEsREYmVQgEgkWBLyVSqW1bEXYmISKwUCpHWqplMTq2loaUj\n7lJERGKjUIgk6mYzxTaydN2WuEsREYmNQiFSOflgkub8+Dd/jrsUEZHYKBQio6ceDMBp43fEXImI\nSHz2GApmtp+ZFUe3TzKzL5hZde5LG1yJujmkMWyzDksVkfzVn5bCb4CUmc0CfgbMAH6Z06riUFTG\n1qJJVO94Q2MgiUje6k8opN29C/gw8EN3/zIwIbdlxaO5eg4z06uob2yPuxQRkVj0JxQ6zexC4OPA\nH6J5hbkrKT4F4w9mum1g8eqNcZciIhKL/oTCpcCxwDXuvtzMZgC357aseFTPmEvSnPrlL8ZdiohI\nLAr2tIC7vwp8AcDMaoBKd78214XFoXzyYQB0rF0IfDDeYkREYtCfo48eNrMqMxsNvAjcZmbf78/K\nzexUM1tsZkvM7Iosj081s4fM7Hkze8nMTt/7tzCARs+knUI61r0caxkiInHpz+6jUe7eCJwF3Obu\nbwdO2dOTzCwJ3ACcBhwEXGhmB/Va7BvA3e5+OHAB8F97U/yASxawvXwms1hFU3tXrKWIiMShP6FQ\nYGYTgPPY2dHcH0cBS9x9mbt3AHcBZ/ZaxoGq6PYoYN1erD8nUmMP5kBbycI1DXGXIiIy6PoTClcD\nfwaWuvszZjYTeKMfz5sErM64vyaal+kq4CIzWwPcD/xTP9abU1UzjqTOGlm69PW4SxERGXR7DAV3\n/7W7H+bun4vuL3P3s/uxbsu2ul73LwTmu/tk4HTgF2b2pprM7DIzW2BmCzZt2tSPl9535TPmAdC0\nYkFOX0dEZCjqT0fzZDP7nZltNLN6M/uNmU3ux7rXAFMy7k/mzbuHPgncDeDuTwAlQG3vFbn7ze4+\nz93n1dXV9eOl34Jxh5AmQfHGl3L7OiIiQ1B/dh/dBtwLTCTs/rkvmrcnzwCzzWyGmRUROpLv7bXM\nKuBkADM7kBAKuW0K7ElRGdvKZzKt/XW2NOnMZhHJL/0JhTp3v83du6JpPrDHr+vR0BiXE/ojFhGO\nMnrFzK42szOixb4KfNrMXgTuBC7xITDw0LOd0zgksZwXVm2LuxQRkUG1x5PXgM1mdhHhQxtCP0C/\nrkTj7vcTOpAz530z4/arwPH9K3XwnHTSeyn6ywMsfmMxJx80Pu5yREQGTX9aCp8gHI66AVgPnEMY\n+mLEKpp6JACty5+OuRIRkcHVn6OPVrn7Ge5e5+5j3f1DhBPZRq7xh9JpRdRsfZ72rlTc1YiIDJp9\nvfLaVwa0iqGmoIgdow9lLotZuHZ73NWIiAyafQ2FbOcgjCglM4/lEFvO80s3xF2KiMig2ddQiP0I\noVwr2+84iizFo3//W9yliIgMmj6PPjKzHWT/8DegNGcVDRWTjwLgkPRrdKbSFCb3NT9FRIaPPj/p\n3L3S3auyTJXu3p9DWYe3ijqay6fxNl/ESxocT0TyhL7+7kbBfu/g6MRrPP6GLs8pIvlBobAbxbNO\npMpaWPuazlcQkfygUNid6ScAULH+SZp10R0RyQMKhd2pmkhr5XSOTizisSWb465GRCTn+jN09g4z\na+w1rY6G0545GEXGqWjWiRydeI1HXov9onAiIjnXn5bC94F/JgybPRn4GnAL4fKat+autKEhOetd\nVFkL9YseZwgM4CoiklP9CYVT3f0n7r7D3Rvd/WbgdHf/FVCT4/riN/MknASHti3glXWNcVcjIpJT\n/QmFtJmdZ2aJaDov47GR/9W5tIauiW/nxMSLfOYXukSniIxs/QmFjwIXAxuj6WLgIjMrJVxEZ8Qr\n3P+9HJZYzoSC5rhLERHJqf4Mnb3M3T/o7rXR9EF3X+Lure7+6GAUGbtZp5DAmbL1cd6o3xF3NSIi\nOdOfo48mR0cabTSzejP7jZlNHozihowJc0mVj+O9yWf548vr465GRCRn+rP76DbgXmAi4Qik+6J5\n+SORIHnQBzkx+SLzH3lVRyGJyIjVn1Coc/fb3L0rmuYDdTmua+g54AOU0c5RqRdYuFZHIYnIyNSf\nUNhsZheZWTKaLgK25LqwIWf6CaRLqjm9YAG/eW5N3NWIiOREf0LhE8B5wAZgPXAOcGkuixqSkoUk\nDng/7yt4jj+/sILOVDruikREBlx/jj5a5e5nuHudu4919w8BZw1CbUPPoedQmm7mbW1P88Ci+rir\nEREZcPs6IN5XBrSK4WLGiXj5WC4oeZLbn1wVdzUiIgNuX0PBBrSK4SKRxA45m3f4s7y0ZCVLNzXF\nXZGIyIDa11DI32My33Y+Se/kjOTjfOSWJ+OuRkRkQPUZCn0Mmd1oZjsI5yzkpwlzYfyhfKbyMRpb\nu9jW3BF3RSIiA6bPUHD3SnevyjJVunvBYBY5pJjBER9nStvrzOxawu1Proy7IhGRAZPTK6+Z2alm\nttjMlpjZFVke/4GZvRBNr5tZQy7rGTCHnguW4EuFv+O6B96gSZfqFJERImehYGZJ4AbgNOAg4EIz\nOyhzGXf/srvPdfe5wI+A3+aqngFVWg2HX8zJhQupSDdy08NL465IRGRA5LKlcBSwJBpltYNwpbYz\nd7P8hcCdOaxnYB39WRKpNq6evIBb/rGMdQ2tcVckIvKW5TIUJgGrM+6viea9iZlNA2YAD+awnoE1\n7iAoqeb0rT/HUx188Ef5MYq4iIxsuQyFbOcy9HUo6wXAPe6eyrois8vMbIGZLdi0adOAFfiWnfMz\nCtLtXH/AIrY0d/DSmuHRJSIi0pdchsIaYErG/cnAuj6WvYDd7Dpy95vdfZ67z6urG0IDtO53Mkw8\ngvdu/SUliRQfueUpUun8PYVDRIa/XIbCM8BsM5thZkWED/57ey9kZvsDNcATOawlN8zgxK+T2L6S\nO0b9hKb2Lm57bHncVYmI7LOchYK7dxGu4fxnYBFwt7u/YmZXm9kZGYteCNzlw/XKNXNOhclHcURy\nCRNKu7jm/kUa/kJEhi0bbp/F8+bN8wULFsRdxq5WPw0/ew/Nx3yV4546hpl15dzz2eNIJvJziCgR\nGXrM7Fl3n7en5XJ68lremHIUlNVS/tQP+NZ7anl+VQMnfuehuKsSEdlrCoWB8ukHIVHI++pv5v2H\nTWBdQyuPL9kcd1UiIntFoTBQaqZBeS324p1859hOigoSfOzWp1mm/gURGUYUCgPp/3kSKidS9qev\n8KfLj2VUaSEX/+xpNmxvi7syEZF+USgMpJIqeP93oX4h0+98J//9iaNYv72Vd333YRpaNMS2iAx9\nCoWBdsD74eCzoHEth7CE2z91NG2dKY6/9kG2NLXHXZ2IyG4pFHLhA98HS8Jtp3PchAQ//fg8WjpT\nHP+tB1mzrSXu6kRE+qRQyIXSGvj4feBp+PUlnDxnNAeMr6Qz5Zx94+O8tqEx7gpFRLJSKOTK1KPh\nAz+E5Y/ADw/lf7/4Tv74hRPY2tzB+69/lGdWbI27QhGRN1Eo5NLhH4Xj/gl2rIenb+GA8VU8+NWT\nKEwa5930BDc+vFQD6InIkKJQyLVT/g1KR8P9X4MXfsmU0WU8eeXJnHboeL71p9eY+29/YfVW9TOI\nyNCgUMi1RBK+sghKquH3n4OXfk11WRE3fOQIvnfu22ju6OLE7zzEbY8tV6tBRGKnUBgMhSUhGKa/\nA353GSz8LWbG2W+fzD/+z7upLCnk3+57lXNuepyFa7fHXa2I5DGNkjqYOprhewdAeyOc9ws4KIwg\n7u78/oW1/POvX6Ir7dSUFXL7p47m4ImjYi5YREYKjZI6FBWVw1deheJKuPtieO7nAJgZHz58Ms/+\n3/cwqbqUxrYu3n/9oxx+9V94eY1aDiIyeNRSiEP7Drj747D0ARg1Bb70criKW2R7aye3PbacHz24\nhFTaqSgu4NqzD+V9B4+nMKkcF5G919+WgkIhLqlOuO+L8MIdcNgF8IEfQFHZLos0tnXygesfpb6x\njfauNAaMqyrmpx8/koMnVmGmi/iISP8oFIYDd3jk2/Dwf8C4Q+C8n8OY/d60WCrtPPL6Rr5694s0\ntHTiQMJgck0p93zuOMZWlgx+7SIyrCgUhpM3/gq//TSkU3D6d+Cw83fZnZSpoaWDD93wGKu2ttB9\nBGt1aSGjy4u46zPHKCBEJCuFwnDTsAru+SSseTocuvr+70Hd/rt9yhk/epTNTe1sbuqgI5UGoLwo\nSWcqzayxldz3TyfoOtEiAigUhqd0Gp6bD3+7CtoaoWoSXP7Mm/oaenN3PvijR2lo7aShpZOm9i4A\nChJGVUkBzR0p9h9fyf98/nj1Q4jkKYXCcNa0Cf76TXjxl5AshnPnw/6n9blLqbeGlg7OufFxGlo7\naWzt6mlFFCYNd0gkjJm15fzqM+HqcCIy8ikURoIVj8Ed50BnC0w7AU65CqYcuVercHc+dMNjNLZ2\n0tjWxdaWDjJ/5QmDZMKYXFPKjz9yBLPHVlJUoMNeRUYahcJI0dUBz86HP/8LpDvhgA/Au/8vjD1g\nn1fZ0NLBy2u3c+VvX2b99rY3jblUVpSkvKiAxrZOpo8pZ/4njmR8VYl2PYkMYwqFkaa9CZ68ER67\nDjp2QPlY+PQDUD31La/a3Vm+uZnLfr6AVVtbKClM0tKRoisjLLr7q0eXF1FSmGRzUzuzx1bw688e\nR0lh8i3XICK5pVAYqZq3wKPfhyduABwOOSdcs2Hi3AF9GfdwlbhFG3YwrrKY1s4Um5s6SBh0pnb9\nmykuSNCVduoqQmDUN7YxZ1wld112LKVFCgyRoUChMNJtXwM/ey/s2ACegpJRcPbPYNYp/e6Q3lfN\n7V2c/5MnWLKpiTHlRbR1ptnW0gFA79G/EwYOlBcVUJg0mtq7MDMmjiqhMJmgMGnceNHbqa0opry4\nIKd1i+QzhUK+aNse+hwe/HdIdcDYg+DIT8Fh54WB9waRu/Ph/3qcxRsacYfaymK6Umk2N3VQVpSk\nM+W0dabY3V+c2a4BMraymMJkgg2NbZgZc8ZWcMenj6G8KKk+DpG9MCRCwcxOBa4DksBP3f3aLMuc\nB1xF+EL5ort/ZHfrVCj0oasDFv4GnrwBNrwMloQjPgbzLoUJb4u7ul2ce9PjdKWczlSapZuamDCq\nlM5Umg2NbbjT7wApLkhQkDTaO9OYwZjyIpKJBJub2plUXUoyYazZ1oKZMXtsBQUJI5kw7v7scYP2\nXkWGithDwcySwOvAe4A1wDPAhe7+asYys4G7gXe7+zYzG+vuG3e3XoXCHrjD2mfhzguhZTN4Gooq\n4LRvwcFn7fFEuKGmK5Xm3JueYHH9Dtx9lwCpKimkM5Wmqb0L99DK6M/F6xIGBYkEXekwyGBlSSHJ\nhNHY1kltRTEJMzY3tQNhfKlkwli9rRUD5oyrIGnG4vodHDxxFL/6zLE5ff8iA2UohMKxwFXu/r7o\n/pUA7v6fGct8G3jd3X/a3/UqFPZC6zZ48VfwwFXQ2RpaD2Wj4bRvh76Hkqq4KxxwXak05//kCbrS\nTirtLN3UhAMTR5WSSjvrG9sYXVZIKu1siwYXLIk6yju60v0OFgghlDQj5Y4BZUUFJBPQ3J6iuiwE\nzbbmDjBjfFUImw2NbUypKSNhsGpraMXMrC3HzEgY/OTieRQXJCguTFCUTFCgodJlgAyFUDgHONXd\nPxXdvxg42t0vz1jm94TWxPGEXUxXufufsqzrMuAygKlTp7595cqVOal5xHKHlY+HYbpf+hWkuwCD\nOe+Dgz8czpYu0VXeup130+O8ur4RB/arrSCVdpZtDuEyYVQJqTRsaGxjTHkRqbSztTl0spcVJUm5\n09qRoiCRIOU+INfdTkRBVZgCJgZ1AAAQ6UlEQVQ0EmZ0RmeolxaGfpXWji7KiwtImPUMcVJTXkTC\nYFtzB3WVJSQMNu5ox4BJNaUkLOxamzamnITBii0tAMyqq+CGjx5BUUEIpcLun0lTH84wNxRC4Vzg\nfb1C4Sh3/6eMZf4AdALnAZOBfwCHuHtDX+tVS+EtSqdgzTOw6D54+hZItaOAyK3zbnqclDvpNLxe\nv4MZteWkHZZvaQZ3JteU4e6saWhlQlUJaQ+h40BteRFphy3N7VSXFuE4DS2dAFQUF5B2p7m9i9Ki\ncLu7H6YgYaSdAQmlbkZoHblDQdIwjM5oF1xxQRIzaOtMAVBeXIABzR0pKkvC7R1tIbCqywoxjIbW\nDkaXF2EYW5vD7rq6ymIANjV1MLayGCOEGcD4USUYsH57G5OqSzGDtQ2tAEypKcMMVm8NQQewcmsL\nBsyoLceAZZub2a+ugusumEtBIkEiEXYjJhMW+puS4WfCrKf/aSQF4VAIhf7sProJeNLd50f3HwCu\ncPdn+lqvQmEAucOaBXD3x6Bly86AmP1emP2eMNVMj7tKeYtS6RAWbZ0pPjn/GdIOaXeWbW5m2ugy\n0g6rtjbjwKTq0tAt1dDK+KoSHGdDYzu4M6aiGHdnS3PHmwKqsqQAd9gRtVTKipK4Q0tHFyWF4XZ7\nVwiswkQCx+lKOYmE4e793mUXl4SFy+am0mFXYUEyhEVXyne5DfRcHbEzld55Ox1ad8UFCQyjvSuE\nZ2lREmNnaw9CkAJURWHa2NbVM0ZZXWUxf/rSO/fpPfQ3FHJ5YPgzwGwzmwGsBS4Aeh9Z9HvgQmC+\nmdUCc4BlOaxJMpmFsZS+uiiM0Lr2WXjld/DaH+CNP4dlCkrDEUyzToFpx0Ohrtcw3CQTRnlxAeXF\nBfz+8hPiLmeP3J3zf/IEDvzik0dz0U+f4rUNjcwZV4k7vL5xB7PrKnDgjY1NAMysLceB5ZubmT6m\nDHdYsaUZgCmjy8Bh1bYWJleX4kQtDA+7A53Q+hhXVYK7Ux+1TOoqinCHzU3tjC4vBkIgAlSXFgHQ\n0Nqxy22AUaXhY3VbayejSgtw6AnPsqIC3MORd6FFl8CjoyTcwXG6v6i3dYYgSbn37BasLsv9AJa5\nPiT1dOCHhP6CW939GjO7Gljg7vdaaJt9DzgVSAHXuPtdu1unWgqDwB22LAkX/3nkW+FcCBwwmH4C\nzHhnuObDpLdDQVHc1YpIP8S++yhXFAox6GiGFY/C8r+HacNLYb4lQuth2vEw7TiYfOSwO+RVJF8M\nhd1HMlIUlYeO6DnvC/dbtoajmVY+FqZHus9JtBAM044LQTH1aHVaiwwzainIW9e2HVY/HYXE4+F2\n97nI4w/b2ZKYdhyU18Zaqki+0u4jiU9HC6xdsLM1seLRcGY1hI7rQ86CKUfBpHlQdwAk1WAVyTWF\nggwdXR2w/oUQEI9dH860zhzVqLgK5n4kjNE0YS7UzlFQiAww9SnI0FFQFFoGU46CE74cjm7augzW\nPhcOg133PDx9887WhCVCQIw7JEzjD4FxB0NpTbzvQyQPKBRk8JnBmP3CdNi5YV46FQ6DXf9imDa8\nDC/eGQ3JEUkWw37vygiKQ2D0TEjoQj4iA0W7j2TocoemetiwEOpfhsdvgM5m6GzZuUxhGYw9MLQk\nxh0awmLsQVBaHV/dIkOQdh/J8GcGlePDNPuUsOsJoLMNNr0G9a9A/UJ4/o6wKyqznyJZDLNODq2J\nsQfAmNmhZVJUHstbERku1FKQkcEddqzf2aqofwUW3x+GDM+ULIKpx0Lt7NChXTsbaveHqok5v4yp\nSJzUUpD8YhY+2Ksmwpz37pzf2QpblsKWN2DzktBvseUNWHBbuLZ1z/OTUFgKc06FMbOiKer30Al4\nkkcUCjKyFZaGfobxh+w63x2aNsLmxbD5ddj0OmxdGs6vWHjPrssmCsN6DjozIzBmwegZUFA8eO9F\nZBAoFCQ/mUHluDDN6DUUcVc7bFsRtSqi6ZXfw/O3s0u/BYS+i8JSOOy8jNbFLBg1RUdFybCkPgWR\nvdHWGFoUW5buDIzF/wsdTb0WtBAWBSVwxMW7tjDK69R/IYNOfQoiuVBSBRMPD1Mmd2jetGvrYstS\nWPogPHbdrstaMoRFYQkcflG4kFHNjLA7qmqyzuaWWOmvT2QgmEHF2DBNO27Xx9Ip2L56Z1BsWQov\n3x2G++gdGFjopygogUPODkFRMwNqpkH1VHV6S85p95FInNIpaFwH25aHfoyty+G5n0Pr1p3DfmRK\nJEOrpKQaDjs/BEX1VKieEoVGtXZNSVYaEE9kJGjdFoKiYdXOaeE90dXwyB4clgjhcOi5GaERTaU1\nCo08pVAQGencQ2g0rNw1NF6+B7rawlFUmedidCssC5dT7d3KqJ4GZWMUGiOUOppFRjozKBsdpsyO\n79O/E372hMaq0KfRHRrbVsLyR3YdQ6pnnYkwJEhmWIyaEgKjemroM1FojGgKBZGRapfQmJt9mdaG\njMDo/rkSlj0Ey9p3HaU2rDR0gk87NoRF1aSdZ5J33y6pyvlbk9xRKIjks9LqMI0/NPvj7TtCWGzP\nCIzu8Fj+D0h3vvk5lgwn8XUHReWEXUOjalIIKrU4hiSFgoj0rbgSxh0Upmy62mHHhnAEVePa6Oc6\n2BH9fPnXkOp48/OSxVA1YWdQVE7YNTSqJkDFOJ0VHgOFgojsu4LicA5FzbS+l0mnwjhTuwTH2jCq\nbeM6WHRf6BjPJnP3VOXEXruqJoQw0fhTA0qhICK5lUhGrYIJwNuzL+MOLVt3DY3GdVFwrIXX/xJa\nHNmOpkoUhpZM1l1V0aTraPSbQkFE4mcG5WPCNOGwvpdra8weGo3rYNnD0fUzshxmXzKq7/6Nqmie\nTvwDFAoiMpyUVIVp7AF9L9PZurNvo/euqsa1sOLvkMrSQV5YlqV/Y+Kuu6/Ka0d8P4dCQURGlsLS\nnRdI6kuq880d5Jmtjld+F3WQZ2l1lI8NneAVdeFnefSzYuyut0tHQyKRs7eZKwoFEck/ycLo5Lwp\nfS+TToeRbzNDo6k+dJo3bYTmjbD5jXA71Z59HRXjdw6UWDEuTJXjM26PC8sUleXmfe6DnIaCmZ0K\nXAckgZ+6+7W9Hr8E+A6wNpr1Y3f/aS5rEhHpl0Ri54WYJh3R93LuYSyq5k1RYNRHtzMCpKke6l8N\nQfKmEwKB4qoQElUTwm6qyvFhl1V53c5AKa8LfSM57vfIWSiYWRK4AXgPsAZ4xszudfdXey36K3e/\nPFd1iIjklNnOkwBrZ+9+2XQ6jIC7YwM0bYAd9eFn08Zo99V6WPl4OFkw266rmpnwxedz8ja65bKl\ncBSwxN2XAZjZXcCZQO9QEBHJD4lE6KwurwUO6Xu57vDYpeWxEWaemPMScxkKk4DVGffXAEdnWe5s\nM3sn8DrwZXdf3XsBM7sMuAxg6tSpOShVRGQIyQyPvs4mz9VL53Dd2XZ89W4P3QdMd/fDgL8B/51t\nRe5+s7vPc/d5dXV1A1ymiIh0y2UorAEyu/YnA+syF3D3Le7e3W1/C32e7igiIoMhl6HwDDDbzGaY\nWRFwAXBv5gJmNiHj7hnAohzWIyIie5CzPgV37zKzy4E/Ew5JvdXdXzGzq4EF7n4v8AUzOwPoArYC\nl+SqHhER2TNdjlNEJA/093Kcw+8cbBERyRmFgoiI9FAoiIhIj2HXp2Bmm4CV+/j0WmDzAJYzUmi7\nZKftkp22S3ZDfbtMc/c9nug17ELhrTCzBf3paMk32i7Zabtkp+2S3UjZLtp9JCIiPRQKIiLSI99C\n4ea4CxiitF2y03bJTtsluxGxXfKqT0FERHYv31oKIiKyGwoFERHpkTehYGanmtliM1tiZlfEXU+c\nzGyFmb1sZi+Y2YJo3mgz+6uZvRH9rIm7zlwzs1vNbKOZLcyYl3U7WHB99Pfzkpnt5qK9w1cf2+Qq\nM1sb/b28YGanZzx2ZbRNFpvZ++KpOvfMbIqZPWRmi8zsFTP7YjR/xP295EUoZFwv+jTgIOBCMxvc\nyxkNPe9y97kZx1VfATzg7rOBB6L7I9184NRe8/raDqcBs6PpMuDGQapxsM3nzdsE4AfR38tcd78f\nIPofugA4OHrOf0X/ayNRF/BVdz8QOAb4fPT+R9zfS16EAhnXi3b3DqD7etGy05nsvPLdfwMfirGW\nQeHufycM2Z6pr+1wJvBzD54EqntdD2RE6GOb9OVM4C53b3f35cASwv/aiOPu6939uej2DsK1XyYx\nAv9e8iUUsl0velJMtQwFDvzFzJ6Nrn8NMM7d10P4BwDGxlZdvPraDvn+N3R5tBvk1oxdi3m5Tcxs\nOnA48BQj8O8lX0KhP9eLzifHu/sRhCbu583snXEXNAzk89/QjcB+wFxgPfC9aH7ebRMzqwB+A3zJ\n3Rt3t2iWecNi2+RLKOzxetH5xN3XRT83Ar8jNPnru5u30c+N8VUYq762Q97+Dbl7vbun3D1NuJZ6\n9y6ivNomZlZICIQ73P230ewR9/eSL6Gwx+tF5wszKzezyu7bwHuBhYTt8fFosY8D/xNPhbHrazvc\nC3wsOqrkGGB7926Dka7XvvAPE/5eIGyTC8ys2MxmEDpVnx7s+gaDmRnwM2CRu38/46ER9/eSs2s0\nDyV9XS865rLiMg74XfgbpwD4pbv/ycyeAe42s08Cq4BzY6xxUJjZncBJQK2ZrQH+FbiW7NvhfuB0\nQmdqC3DpoBc8CPrYJieZ2VzC7o8VwGcAomuu3w28Sjg65/Punoqj7kFwPHAx8LKZvRDN+xdG4N+L\nhrkQEZEe+bL7SERE+kGhICIiPRQKIiLSQ6EgIiI9FAoiItJDoSAiIj0UCiL9YGZzew0ZfcZADcFu\nZl8ys7KBWJfIW6XzFET6wcwuAea5++U5WPeKaN2b9+I5yRF8opjESC0FGVHMbHp0IZRboouh/MXM\nSvtYdj8z+1M0Wuw/zOyAaP65ZrbQzF40s79HQ6NcDZwfXWTmfDO7xMx+HC0/38xujC7CsszMToxG\nE11kZvMzXu9GM1sQ1fVv0bwvABOBh8zsoWjehRYugrTQzL6V8fwmM7vazJ4CjjWza83s1Wj00u/m\nZotK3nF3TZpGzARMJwy5MDe6fzdwUR/LPgDMjm4fDTwY3X4ZmBTdro5+XgL8OOO5PfcJF6a5izAy\n5plAI3Ao4UvXsxm1jI5+JoGHgcOi+yuA2uj2RMJwCXWEYUgeBD4UPebAed3rAhazs7VfHfe21zQy\nJrUUZCRa7u7d49M8SwiKXURDIB8H/Doay+YnQPfAb48B883s04QP8P64z92dECj17v6yh1FFX8l4\n/fPM7DngecLVyrJd/e9I4GF33+TuXcAdQPfQ5inCKJ0QgqcN+KmZnUUYX0fkLcuLAfEk77Rn3E4B\n2XYfJYAGd5/b+wF3/6yZHQ28H3ghGgyuv6+Z7vX6aaAgGkX0a8CR7r4t2q1UkmU92cbh79bmUT+C\nh0EejwJOJoz6eznw7n7UKbJbailIXvJwgZTlZnYu9Fxo/W3R7f3c/Sl3/yawmTAu/g6g8i28ZBXQ\nDGw3s3GECxx1y1z3U8CJZlYbXe/4QuCR3iuLWjqjPFwv+UuEC+CIvGVqKUg++yhwo5l9Aygk9Au8\nCHzHzGYTvrU/EM1bBVwR7Wr6z719IXd/0cyeJ+xOWkbYRdXtZuB/zWy9u7/LzK4EHope/353z3Zt\ni0rgf8ysJFruy3tbk0g2OiRVRER6aPeRiIj00O4jGfHM7AbClbMyXefut8VRj8hQpt1HIiLSQ7uP\nRESkh0JBRER6KBRERKSHQkFERHr8/3a2RgM47E1rAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13d65a90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('2_nestimators.csv')\n",
    "        \n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(0, cvresult.shape[0])\n",
    "        \n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators5_1_219.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#获取的参数n_estimators = 218\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "def modelfit(alg, X_train, y_train, useTrainCV=True, cv_folds=None, early_stopping_rounds=100):\n",
    "    \n",
    "    if useTrainCV:\n",
    "        xgb_param = alg.get_xgb_params()\n",
    "        xgb_param['num_class'] = 3\n",
    "        \n",
    "        xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "        cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "                         metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "        \n",
    "        n_estimators = cvresult.shape[0]\n",
    "        alg.set_params(n_estimators = n_estimators)\n",
    "        \n",
    "        print cvresult\n",
    "        #result = pd.DataFrame(cvresult)   #cv缺省返回结果为DataFrame\n",
    "        #result.to_csv('my_preds.csv', index_label = 'n_estimators')\n",
    "        cvresult.to_csv('my_preds5_1_219.csv', index_label = 'n_estimators')\n",
    "        \n",
    "        # plot\n",
    "        test_means = cvresult['test-mlogloss-mean']\n",
    "        test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "        train_means = cvresult['train-mlogloss-mean']\n",
    "        train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "        x_axis = range(0, n_estimators)\n",
    "        pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "        pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "        pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "        pyplot.xlabel( 'n_estimators' )\n",
    "        pyplot.ylabel( 'Log Loss' )\n",
    "        pyplot.savefig( 'n_estimators5_1_219.png' )\n",
    "    \n",
    "    #Fit the algorithm on the data\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    train_predprob = alg.predict_proba(X_train)\n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "        \n",
    "    #Print model report:\n",
    "    print (\"logloss of train :\" )\n",
    "    print logloss"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
